{
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.12",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [],
   "dockerImageVersionId": 30616,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook",
   "isGpuEnabled": true
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:45.305861Z",
     "iopub.execute_input": "2023-12-12T09:38:45.306130Z",
     "iopub.status.idle": "2023-12-12T09:38:50.040232Z",
     "shell.execute_reply.started": "2023-12-12T09:38:45.306106Z",
     "shell.execute_reply": "2023-12-12T09:38:50.039322Z"
    },
    "trusted": true
   },
   "execution_count": 1,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": "sys.version_info(major=3, minor=10, micro=12, releaselevel='final', serial=0)\nmatplotlib 3.7.4\nnumpy 1.24.3\npandas 2.1.3\nsklearn 1.2.2\ntorch 2.0.0\ncuda:0\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 数据准备"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "!wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:50.042232Z",
     "iopub.execute_input": "2023-12-12T09:38:50.042998Z",
     "iopub.status.idle": "2023-12-12T09:38:51.607893Z",
     "shell.execute_reply.started": "2023-12-12T09:38:50.042962Z",
     "shell.execute_reply": "2023-12-12T09:38:51.606716Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "text": "--2023-12-12 09:38:50--  https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\nResolving storage.googleapis.com (storage.googleapis.com)... 74.125.128.207, 108.177.96.207, 108.177.119.207, ...\nConnecting to storage.googleapis.com (storage.googleapis.com)|74.125.128.207|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 1115394 (1.1M) [text/plain]\nSaving to: ‘shakespeare.txt’\n\nshakespeare.txt     100%[===================>]   1.06M  2.49MB/s    in 0.4s    \n\n2023-12-12 09:38:51 (2.49 MB/s) - ‘shakespeare.txt’ saved [1115394/1115394]\n\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.609700Z",
     "iopub.execute_input": "2023-12-12T09:38:51.610646Z",
     "iopub.status.idle": "2023-12-12T09:38:51.618734Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.610604Z",
     "shell.execute_reply": "2023-12-12T09:38:51.617758Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "text": "length 1115394\nFirst Citizen:\nBefore we proceed any further, hear me speak.\n\nAll:\nSpeak, speak.\n\nFirst Citizen:\nYou\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 构造字典"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.621910Z",
     "iopub.execute_input": "2023-12-12T09:38:51.622640Z",
     "iopub.status.idle": "2023-12-12T09:38:51.645793Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.622606Z",
     "shell.execute_reply": "2023-12-12T09:38:51.644903Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "text": "65\n['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "for idx,char in enumerate(['how','are','you']):\n",
    "    print(idx,char)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.648171Z",
     "iopub.execute_input": "2023-12-12T09:38:51.648710Z",
     "iopub.status.idle": "2023-12-12T09:38:51.657263Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.648669Z",
     "shell.execute_reply": "2023-12-12T09:38:51.656414Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "text": "0 how\n1 are\n2 you\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.658474Z",
     "iopub.execute_input": "2023-12-12T09:38:51.658791Z",
     "iopub.status.idle": "2023-12-12T09:38:51.671523Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.658766Z",
     "shell.execute_reply": "2023-12-12T09:38:51.670564Z"
    },
    "trusted": true
   },
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "text": "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.672974Z",
     "iopub.execute_input": "2023-12-12T09:38:51.673266Z",
     "iopub.status.idle": "2023-12-12T09:38:51.683131Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.673242Z",
     "shell.execute_reply": "2023-12-12T09:38:51.682347Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "text": "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.684641Z",
     "iopub.execute_input": "2023-12-12T09:38:51.685600Z",
     "iopub.status.idle": "2023-12-12T09:38:51.876820Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.685573Z",
     "shell.execute_reply": "2023-12-12T09:38:51.875886Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "text": "(1115394,)\n1115394\n[18 47 56 57 58  1 15 47 58 47]\nFirst Citi\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1 #一个样本的长度\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len #样本的个数\n",
    "        \n",
    "    def __getitem__(self, index):#index是样本的索引，返回的是一个样本\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self): #返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "#batch是一个列表，列表中的每一个元素是一个样本，有101个字符，前100个是输入，后100个是输出\n",
    "def collat_fct(batch):\n",
    "    src_list = []\n",
    "    trg_list = []\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1]) #输入\n",
    "        trg_list.append(part[1:]) #输出\n",
    "        \n",
    "    src_list = np.array(src_list) #把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list) #把列表转换为ndarray\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64) #返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "        \n",
    "\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.878036Z",
     "iopub.execute_input": "2023-12-12T09:38:51.878351Z",
     "iopub.status.idle": "2023-12-12T09:38:51.890250Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.878323Z",
     "shell.execute_reply": "2023-12-12T09:38:51.889265Z"
    },
    "trusted": true
   },
   "execution_count": 9,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 定义模型"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True) #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x)\n",
    "        output, hidden = self.rnn(x, hidden) #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        x = self.fc(output)\n",
    "        return x, hidden #x的shape是(batch_size, seq_len, vocab_size)\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "sample_inputs = torch.randint(0, vocab_size, (2, 128))\n",
    "    \n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))       \n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")\n",
    "    \n",
    "CharRNN(vocab_size)(sample_inputs)[0].shape"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:51.891489Z",
     "iopub.execute_input": "2023-12-12T09:38:51.891761Z",
     "iopub.status.idle": "2023-12-12T09:38:52.073035Z",
     "shell.execute_reply.started": "2023-12-12T09:38:51.891738Z",
     "shell.execute_reply": "2023-12-12T09:38:52.072125Z"
    },
    "trusted": true
   },
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": "=================================== 一层单向 RNN ===================================\n            embedding.weight            paramerters num: 16640\n            rnn.weight_ih_l0            paramerters num: 262144\n            rnn.weight_hh_l0            paramerters num: 1048576\n             rnn.bias_ih_l0             paramerters num: 1024\n             rnn.bias_hh_l0             paramerters num: 1024\n               fc.weight                paramerters num: 66560\n                fc.bias                 paramerters num: 65\n",
     "output_type": "stream"
    },
    {
     "execution_count": 10,
     "output_type": "execute_result",
     "data": {
      "text/plain": "torch.Size([2, 128, 65])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 训练"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:52.076483Z",
     "iopub.execute_input": "2023-12-12T09:38:52.076885Z",
     "iopub.status.idle": "2023-12-12T09:38:52.085855Z",
     "shell.execute_reply.started": "2023-12-12T09:38:52.076858Z",
     "shell.execute_reply": "2023-12-12T09:38:52.084921Z"
    },
    "trusted": true
   },
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_dl, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:38:52.087675Z",
     "iopub.execute_input": "2023-12-12T09:38:52.088141Z",
     "iopub.status.idle": "2023-12-12T09:44:59.444404Z",
     "shell.execute_reply.started": "2023-12-12T09:38:52.088113Z",
     "shell.execute_reply": "2023-12-12T09:44:59.443330Z"
    },
    "trusted": true
   },
   "execution_count": 12,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/17300 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "4632eb31d7dc4c6796e1dedacd72cd49"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:44:59.445688Z",
     "iopub.execute_input": "2023-12-12T09:44:59.445997Z",
     "iopub.status.idle": "2023-12-12T09:44:59.722065Z",
     "shell.execute_reply.started": "2023-12-12T09:44:59.445971Z",
     "shell.execute_reply": "2023-12-12T09:44:59.721124Z"
    },
    "trusted": true
   },
   "execution_count": 13,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ6ElEQVR4nO3dd1hUV/4G8PcODEMdikgvoihWUGxBY0nsmkSTrMlqNqa3NbtmTYw/syUx2UQ3dd0UY6ppxlTNbmLDghU72EVQBFSKgDDUYZg5vz9m5sJIUYrOxXk/z+Mzzp07M2e+XoeXc885VxJCCBAREREpjMreDSAiIiJqDEMKERERKRJDChERESkSQwoREREpEkMKERERKRJDChERESkSQwoREREpEkMKERERKZKzvRtwNUwmEy5cuAAvLy9IkmTv5hAREdFVEEKgrKwMISEhUKla3i/SIULKhQsXEB4ebu9mEBERUSvk5OQgLCysxc/rECHFy8sLgPlDarXadntdg8GADRs2YPz48VCr1e32uh0N68AaAKyBFevAGgCsgVVb66DT6RAeHi7/HG+pDhFSrKd4tFptu4cUd3d3aLVahz8IHb0OrAFrYMU6sAYAa2DVXnVo7VANDpwlIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFcuiQ8vmuLPyUqUJaXpm9m0JERESXceiQsuZoHrblqZBzqcreTSEiIqLLOHRIsV442iSEXdtBREREDTl0SFFJ5pjCjEJERKQ8Dh1SLBmFPSlEREQK5OAhhT0pRERESuXQIUVl6UlhRiEiIlIehw4pHDhLRESkXA4dUjhwloiISLkcOqRYu1IEUwoREZHiOHRIkXtS7NwOIiIiasihQwrHpBARESmXQ4cUjkkhIiJSLocOKXWLudm3HURERNRQm0LK4sWLIUkSnnnmmWb3++GHH9CzZ0+4urqiX79+WLNmTVvett1IHDhLRESkWK0OKfv27cOyZcsQGxvb7H67du3CjBkz8MgjjyAlJQXTpk3DtGnTcPTo0da+dbvhwFkiIiLlalVIKS8vx3333YePP/4Yvr6+ze67ZMkSTJw4EfPmzUOvXr3wyiuvID4+Hu+9916rGtyeOHCWiIhIuVoVUmbPno0pU6Zg7NixV9w3OTm5wX4TJkxAcnJya966XfHaPURERMrl3NInrFy5EgcPHsS+ffuuav+8vDwEBgbabAsMDEReXl6Tz9Hr9dDr9fJ9nU4HADAYDDAYDC1tctMs6aS21ti+r9vBWD87a8Aa1L91VKwDawCwBlZtrUNb69eikJKTk4M5c+YgMTERrq6ubXrj5ixatAgLFy5ssH3Dhg1wd3dvt/e5WKACoMKJkyewpuR4u71uR5WYmGjvJtgda8AaWLEOrAHAGli1tg6VlZVtet8WhZQDBw6goKAA8fHx8jaj0Yht27bhvffeg16vh5OTk81zgoKCkJ+fb7MtPz8fQUFBTb7PggULMHfuXPm+TqdDeHg4xo8fD61W25ImN2tNaQpQfBE9Ynpi8vCodnvdjsZgMCAxMRHjxo2DWq22d3PsgjVgDaxYB9YAYA2s2loH65mQ1mpRSBkzZgyOHDlis+2hhx5Cz549MX/+/AYBBQASEhKwadMmm2nKiYmJSEhIaPJ9NBoNNBpNg+1qtbpdDxZnlbm9KpXKoQ9Cq/aub0fEGrAGVqwDawCwBlatrUNba9eikOLl5YW+ffvabPPw8ECnTp3k7bNmzUJoaCgWLVoEAJgzZw5GjRqFt956C1OmTMHKlSuxf/9+fPTRR21qeHvgYm5ERETK1e4rzmZnZyM3N1e+P2zYMKxYsQIfffQR4uLi8OOPP2L16tUNwo49cDE3IiIi5Wrx7J7LJSUlNXsfAKZPn47p06e39a3aHRdzIyIiUi7HvnaP5ZaLuRERESmPY4cUFRdzIyIiUirHDimWW4YUIiIi5XHokCKPSWFKISIiUhyHDimcgkxERKRcDh1SVNYpyPZtBhERETXCoUOK9SrInN1DRESkPI4dUiy3HJNCRESkPA4dUuoGztq5IURERNSAQ4cUDpwlIiJSLgcPKdZl8ZlSiIiIlMaxQ4rllqd7iIiIlMehQ4o8BZkhhYiISHEcOqRwCjIREZFyOXRI4WJuREREyuXQIUXitXuIiIgUy7FDiuWWU5CJiIiUx6FDCq+CTEREpFwOHVK4mBsREZFyMaSAA2eJiIiUyLFDCni6h4iISKkcOqRwMTciIiLlcuiQUjcmhSmFiIhIaRw8pFgvMEhERERK49ghxXLLMSlERETK49AhRSVfu8fODSEiIqIGHDykmG/ZkUJERKQ8Dh1SeBVkIiIi5XLwkGK+ZUQhIiJSHoYUAIKDUoiIiBTHoUOKilOQiYiIFMuhQ4p1CjLHpBARESmPY4cUa08KMwoREZHiOHhIMd8ypBARESmPQ4cUFacgExERKZaDhxTzLSMKERGR8jh0SOHAWSIiIuVy7JDCgbNERESK5eAhxXzLqyATEREpj0OHFC7mRkREpFwOHVI4JoWIiEi5WhRSli5ditjYWGi1Wmi1WiQkJGDt2rVN7r98+XJIkmTzx9XVtc2Nbi8ck0JERKRczi3ZOSwsDIsXL0b37t0hhMAXX3yBqVOnIiUlBX369Gn0OVqtFmlpafJ9azBQAi7mRkREpFwtCim33367zf1XX30VS5cuxe7du5sMKZIkISgoqPUtvIas66TwdA8REZHytCik1Gc0GvHDDz+goqICCQkJTe5XXl6OyMhImEwmxMfH47XXXmsy0Fjp9Xro9Xr5vk6nAwAYDAYYDIbWNrkBYTIBAEwmU7u+bkdj/eysAWtQ/9ZRsQ6sAcAaWLW1Dm2tnyRaOP/2yJEjSEhIQHV1NTw9PbFixQpMnjy50X2Tk5ORnp6O2NhYlJaW4s0338S2bdtw7NgxhIWFNfkeL730EhYuXNhg+4oVK+Du7t6S5jZr30UJX2c4oYe3CbN7m9rtdYmIiAiorKzEzJkzUVpaCq1W2+Lntzik1NTUIDs7G6Wlpfjxxx/xySefYOvWrejdu/cVn2swGNCrVy/MmDEDr7zySpP7NdaTEh4ejsLCwlZ9yKasOngOz686jpuifPHVw4Pb7XU7GoPBgMTERIwbNw5qtdrezbEL1oA1sGIdWAOANbBqax10Oh38/f1bHVJafLrHxcUF0dHRAICBAwdi3759WLJkCZYtW3bF56rVagwYMAAZGRnN7qfRaKDRaBp9fnseLM7OTjav7ejau74dEWvAGlixDqwBwBpYtbYOba1dm9dJMZlMNr0ezTEajThy5AiCg4Pb+rbtgou5ERERKVeLelIWLFiASZMmISIiAmVlZVixYgWSkpKwfv16AMCsWbMQGhqKRYsWAQBefvll3HTTTYiOjkZJSQneeOMNZGVl4dFHH23/T9IKdYu52bUZRERE1IgWhZSCggLMmjULubm58Pb2RmxsLNavX49x48YBALKzs6FS1XXOXLp0CY899hjy8vLg6+uLgQMHYteuXVc1fuV64LV7iIiIlKtFIeXTTz9t9vGkpCSb+++88w7eeeedFjfqeuGKs0RERMrl0NfusS7mxoxCRESkPA4eUswphSvOEhERKY9DhxReBZmIiEi5HDuk8HwPERGRYjl2SLHccgoyERGR8jh0SKnrSGFKISIiUhqHDinWKcgmXluQiIhIcRw8pJhv2Y9CRESkPI4dUmBdzI0xhYiISGkcOqTIY1KYUYiIiBTHwUMKF3MjIiJSKocOKdYxKZyCTEREpDwMKQA4dJaIiEh5HDukwHq6x84NISIiogYcOqRw4CwREZFyOXRIkThwloiISLEcPKSYbxlRiIiIlMexQ4rllou5ERERKY9DhxTrOinMKERERMrDkAKOSSEiIlIihw4pEmf3EBERKRZDCjhwloiISIkcO6SAp3uIiIiUyqFDChdzIyIiUi6HDil1FxhkSiEiIlIaBw8p0pV3IiIiIrtw7JBiuWVPChERkfI4dEjhYm5ERETK5dAhpW5Min3bQURERA05dEip60lhSiEiIlIahw4pXMyNiIhIuRhSwIGzRERESuTQIYUDZ4mIiJTLoUMKpyATEREpl2OHFC7mRkREpFgOHlLMt5yCTEREpDwOHVI4BZmIiEi5HDqk1I1JsWsziIiIqBEOHVJU9YaksDeFiIhIWRw6pNQfOMuMQkREpCwtCilLly5FbGwstFottFotEhISsHbt2maf88MPP6Bnz55wdXVFv379sGbNmjY1uD3Vn9zDachERETK0qKQEhYWhsWLF+PAgQPYv38/br31VkydOhXHjh1rdP9du3ZhxowZeOSRR5CSkoJp06Zh2rRpOHr0aLs0vq1U9XtS7NgOIiIiaqhFIeX222/H5MmT0b17d/To0QOvvvoqPD09sXv37kb3X7JkCSZOnIh58+ahV69eeOWVVxAfH4/33nuvXRrfVvVXSWFPChERkbK0ekyK0WjEypUrUVFRgYSEhEb3SU5OxtixY222TZgwAcnJya1923bFMSlERETK5dzSJxw5cgQJCQmorq6Gp6cnVq1ahd69eze6b15eHgIDA222BQYGIi8vr9n30Ov10Ov18n2dTgcAMBgMMBgMLW1yk4y1da9VU2OAE0zt9todibWm7VnbjoY1YA2sWAfWAGANrNpah7bWr8UhJSYmBqmpqSgtLcWPP/6IBx54AFu3bm0yqLTGokWLsHDhwgbbN2zYAHd393Z7H70RsJZg3fr10Di120t3SImJifZugt2xBqyBFevAGgCsgVVr61BZWdmm921xSHFxcUF0dDQAYODAgdi3bx+WLFmCZcuWNdg3KCgI+fn5Ntvy8/MRFBTU7HssWLAAc+fOle/rdDqEh4dj/Pjx0Gq1LW1yk3SV1cDebQCAcePHw1PT4nLcEAwGAxITEzFu3Dio1Wp7N8cuWAPWwIp1YA0A1sCqrXWwnglprTb/VDaZTDanZupLSEjApk2b8Mwzz8jbEhMTmxzDYqXRaKDRaBpsV6vV7XqwaNRG+e9Ozs4OfSAC7V/fjog1YA2sWAfWAGANrFpbh7bWrkUhZcGCBZg0aRIiIiJQVlaGFStWICkpCevXrwcAzJo1C6GhoVi0aBEAYM6cORg1ahTeeustTJkyBStXrsT+/fvx0UcftanR7YUDZ4mIiJSrRSGloKAAs2bNQm5uLry9vREbG4v169dj3LhxAIDs7GyoVHUThoYNG4YVK1bgb3/7G1544QV0794dq1evRt++fdv3U7SSxGXxiYiIFKtFIeXTTz9t9vGkpKQG26ZPn47p06e3qFHXi4o9KURERIrl2Nfuqfd3LuZGRESkLI4dUuqf7rFfM4iIiKgRDh5S6lIKe1KIiIiUxaFDCgBI1j4UZhQiIiJFYUix3JoYUoiIiBSFIcWSUni6h4iISFkYUiy3jChERETKwpBiuTXxfA8REZGiMKRIV96HiIiIrj+HDylWHJNCRESkLA4fUqw9KcwoREREysKQYrllTwoREZGyMKRYbhlRiIiIlIUhxXIr2JNCRESkKAwpHJNCRESkSAwpllsuk0JERKQsDh9SwGXxiYiIFMnhQ4q1AMwoREREyuLwIcWKPSlERETK4vAhhcviExERKRNDiuWWPSlERETKwpBiuWVGISIiUhaGFM7uISIiUiSGFMstIwoREZGyMKRYbrksPhERkbI4fEipW8zNvs0gIiIiWw4fUriYGxERkTI5fEix4sBZIiIiZXH4kMKrIBMRESkTQ4rllgNniYiIlIUhxXLLiEJERKQsDh9SwMXciIiIFMnhQwpn9xARESmTw4cUK/akEBERKYvDhxTO7iEiIlImhhTLreDQWSIiIkVhSLHcmkx2bQYRERFdhiHFerrHvs0gIiKiyzCkWG45cJaIiEhZGFI4cJaIiEiRHD6kWHFZfCIiImVpUUhZtGgRBg8eDC8vLwQEBGDatGlIS0tr9jnLly+HJEk2f1xdXdvU6PYkL+Zm11YQERHR5VoUUrZu3YrZs2dj9+7dSExMhMFgwPjx41FRUdHs87RaLXJzc+U/WVlZbWr0tcAxKURERMri3JKd161bZ3N/+fLlCAgIwIEDBzBy5MgmnydJEoKCglrXwmtMkgQACSZmFCIiIkVpUUi5XGlpKQDAz8+v2f3Ky8sRGRkJk8mE+Ph4vPbaa+jTp0+T++v1euj1evm+TqcDABgMBhgMhrY02YbBYJBn99TW1rbra3ck1s/tqJ8fYA0A1sCKdWANANbAqq11aGv9JNHKEaMmkwl33HEHSkpKsGPHjib3S05ORnp6OmJjY1FaWoo333wT27Ztw7FjxxAWFtboc1566SUsXLiwwfYVK1bA3d29Nc1t0vvHVThVqsL90UYM6szuFCIiovZSWVmJmTNnorS0FFqttsXPb3VIeeqpp7B27Vrs2LGjybDRGIPBgF69emHGjBl45ZVXGt2nsZ6U8PBwFBYWtupDNteWO5dsQlqpCm/e3RdT+4e022t3JAaDAYmJiRg3bhzUarW9m2MXrAFrYMU6sAYAa2DV1jrodDr4+/u3OqS06nTP008/jV9//RXbtm1rUUABALVajQEDBiAjI6PJfTQaDTQaTaPPbe+DxXq6R1I5OfSBCFyb+nY0rAFrYMU6sAYAa2DV2jq0tXYtmt0jhMDTTz+NVatWYfPmzYiKimrxGxqNRhw5cgTBwcEtfu61wGXxiYiIlKlFPSmzZ8/GihUr8Msvv8DLywt5eXkAAG9vb7i5uQEAZs2ahdDQUCxatAgA8PLLL+Omm25CdHQ0SkpK8MYbbyArKwuPPvpoO3+UtuEUZCIiImVpUUhZunQpAGD06NE22z///HM8+OCDAIDs7GyoVHUdNJcuXcJjjz2GvLw8+Pr6YuDAgdi1axd69+7dtpa3E+vpHnalEBERKUuLQsrVjLFNSkqyuf/OO+/gnXfeaVGjrifr6R72pBARESmLw1+7p+4qyHZtBhEREV2GIcVyK3i+h4iISFEYUuTTPfZtBxEREdliSLH+hWNSiIiIFIUhxXLLnhQiIiJlYUixLubGnhQiIiJFcfiQYsWeFCIiImVx+JBSN7uHiIiIlIQhhad7iIiIFIkhxXLLFWeJiIiUhSFF7kmxbzuIiIjIFkOK5ZYDZ4mIiJSFIcVyy2XxiYiIlMXhQ4oVT/cQEREpi8OHFBVn9xARESmSw4cUK45JISIiUhaHDymc3UNERKRMDCmWW66TQkREpCwMKZZbjkkhIiJSFoYU6+ke+zaDiIiILsOQYrnl6R4iIiJlYUix3DKjEBERKYvDhxRrSuEUZCIiImVx+JDCZfGJiIiUiSHFcsvTPURERMrCkMJl8YmIiBSJIcVyyzEpREREysKQYrnlFGQiIiJlYUjhtXuIiIgUiSHFcssxKURERMrCkGK5ZUQhIiJSFocPKXWLuTGmEBERKYnDhxTJ0ofCjEJERKQsDCmWW05BJiIiUhaGFGtK4agUIiIiRWFIsdyaTHZtBhEREV2GIcWSUowclEJERKQoDh9SNJYKVNUY7dsQIiIisuHwIcXVyXyrqzbYtyFERERkgyHF2XxbVl1r34YQERGRjRaFlEWLFmHw4MHw8vJCQEAApk2bhrS0tCs+74cffkDPnj3h6uqKfv36Yc2aNa1ucHtzs/SklLEnhYiISFFaFFK2bt2K2bNnY/fu3UhMTITBYMD48eNRUVHR5HN27dqFGTNm4JFHHkFKSgqmTZuGadOm4ejRo21ufHtwdTIPmC3XsyeFiIhISZxbsvO6dets7i9fvhwBAQE4cOAARo4c2ehzlixZgokTJ2LevHkAgFdeeQWJiYl477338OGHH7ay2e3HVe5JYUghIiJSkjaNSSktLQUA+Pn5NblPcnIyxo4da7NtwoQJSE5Obstbtxs3S0yrrDGi1sjFUoiIiJSiRT0p9ZlMJjzzzDMYPnw4+vbt2+R+eXl5CAwMtNkWGBiIvLy8Jp+j1+uh1+vl+zqdDgBgMBhgMLTf2BGDwSD3pADApfJq+Lir2+31OwprTduzth0Na8AaWLEOrAHAGli1tQ5trV+rQ8rs2bNx9OhR7Nixo00NaMyiRYuwcOHCBts3bNgAd3f3dn0vZxWglgQMQsL/1iWik2u7vnyHkpiYaO8m2B1rwBpYsQ6sAcAaWLW2DpWVlW1631aFlKeffhq//vortm3bhrCwsGb3DQoKQn5+vs22/Px8BAUFNfmcBQsWYO7cufJ9nU6H8PBwjB8/HlqttjVNbpTBYEBiYiK07hoUVdRgUMII9Ar2arfX7yisdRg3bhzUasfrSQJYA4A1sGIdWAOANbBqax2sZ0Jaq0UhRQiBP/3pT1i1ahWSkpIQFRV1xeckJCRg06ZNeOaZZ+RtiYmJSEhIaPI5Go0GGo2mwXa1Wn1NDhYvV2cUVdSgqlY49MF4rerbkbAGrIEV68AaAKyBVWvr0NbatSikzJ49GytWrMAvv/wCLy8veVyJt7c33NzcAACzZs1CaGgoFi1aBACYM2cORo0ahbfeegtTpkzBypUrsX//fnz00Udtanh78rKs6MYZPkRERMrRotk9S5cuRWlpKUaPHo3g4GD5z3fffSfvk52djdzcXPn+sGHDsGLFCnz00UeIi4vDjz/+iNWrVzc72PZ689KYQwrXSiEiIlKOFp/uuZKkpKQG26ZPn47p06e35K2uK0+5J8WxR3ETEREpicNfuwcAPC09KTqe7iEiIlIMhhRwTAoREZESMaSg/pgUnu4hIiJSCoYUsCeFiIhIiRhSUDcmhSGFiIhIORhSUL8nhad7iIiIlIIhBfWnILMnhYiISCkYUlA3cJYhhYiISDkYUgD4ursAAIoq9DCZrrxgHREREV17DCkAQn1coXaSUG0w4UJplb2bQ0RERGBIAQA4O6kQ5e8BAMgoKLdza4iIiAhgSJFFB3gCYEghIiJSCoYUi+jODClERERKwpBiER3oBQBIZ0ghIiJSBIYUi/o9KUJwhg8REZG9MaRYdO3sAZUElFYZUFheY+/mEBEROTyGFAtXtRMi/NwBAKfyy+zcGiIiImJIqadnkBYAcCJXZ+eWEBEREUNKPb1DzCHl+AWGFCIiIntjSKmnd7AlpLAnhYiIyO4YUuqx9qRkFJSj2mC0c2uIiIgcG0NKPcHervBxV6PWJLioGxERkZ0xpNQjSVLdKR+OSyEiIrIrhpTL9LGc8jlyvtTOLSEiInJsDCmX6R/uCwA4mH3Jzi0hIiJybAwpl4mP9AEAnMwrQ4W+1r6NISIicmAMKZcJ9nZDiLcrjCaBQ+dK7N0cIiIih8WQ0ogBkeZTPinZJfZtCBERkQNjSGlEfIRlXEoWx6UQERHZC0NKI/qH+wDgDB8iIiJ7YkhpRPdATwBAQZkeumqDnVtDRETkmBhSGqF1VSNQqwEArjxLRERkJwwpTege4AWAIYWIiMheGFKaEB1gPuXDkEJERGQfDClN6MaQQkREZFcMKU3ozpBCRERkVwwpTbCe7sm5VIlqg9HOrSEiInI8DClN6OThAn9PDYQAtpwssHdziIiIHA5DShMkScKMIeEAgA+3noYQws4tIiIiciwMKc14YFgXaJxVOHSuFLvPFNu7OURERA6FIaUZ/p4a3BUfBgD48cA5O7eGiIjIsbQ4pGzbtg233347QkJCIEkSVq9e3ez+SUlJkCSpwZ+8vLzWtvm6unNAKABgw/E86Gs5gJaIiOh6aXFIqaioQFxcHN5///0WPS8tLQ25ubnyn4CAgJa+tV0MivRFoFaDsupabD9ViM92ZGL44s04c5FTk4mIiK4l55Y+YdKkSZg0aVKL3yggIAA+Pj4tfp69qVQSJvcLxuc7z+LL3VnYduoiAOCLXWexcGpfO7eOiIjoxtXikNJa/fv3h16vR9++ffHSSy9h+PDhTe6r1+uh1+vl+zqdDgBgMBhgMLTfVYmtr3Wl15wWF4Tlu87KAQUALpRUtWtb7Olq63AjYw1YAyvWgTUAWAOrttahrfWTRBvm1kqShFWrVmHatGlN7pOWloakpCQMGjQIer0en3zyCb766ivs2bMH8fHxjT7npZdewsKFCxtsX7FiBdzd3Vvb3Db5JUuFzRfqzo510gj8I96ICgNQrAfCPe3SLCIiIsWqrKzEzJkzUVpaCq1W2+LnX/OQ0phRo0YhIiICX331VaOPN9aTEh4ejsLCwlZ9yKYYDAYkJiZi3LhxUKvVze6rNxjx+DcpKC6vwcl883iU/S/cgmd/OIKt6YX44fEh6B/u025tu55aUocbFWvAGlixDqwBwBpYtbUOOp0O/v7+rQ4p1+10T31DhgzBjh07mnxco9FAo9E02K5Wq6/JwXI1r6tWq7HisQQAwIjXNyOnuApHLpRj15kiAEDquTIM7tq53dt2PV2r+nYkrAFrYMU6sAYAa2DV2jq0tXZ2WSclNTUVwcHB9njrdtEn2BsA8O3ebBiM5o6otPwyezaJiIjohtPinpTy8nJkZGTI9zMzM5Gamgo/Pz9ERERgwYIFOH/+PL788ksAwL///W9ERUWhT58+qK6uxieffILNmzdjw4YN7fcprrM+IVqsO5aHDcfz5W3plpBiMJpgEgIaZyd7NY+IiOiG0OKelP3792PAgAEYMGAAAGDu3LkYMGAA/vGPfwAAcnNzkZ2dLe9fU1ODZ599Fv369cOoUaNw6NAhbNy4EWPGjGmnj3D9je0d2GDbqfxymEwCf/zmIAb9cyMKdNV2aBkREdGNo8U9KaNHj272YnvLly+3uf/888/j+eefb3HDlKxXsBa3xQbj18O58rYqgxH7sy4h0dK7sjuzGHfEhdiriURERB0er93TSs+Nj4GLswoBXhp0DzDPP35/S91psLQ8nb2aRkREdEOwy+yeG0EXfw+sf2YkXJxV+Nfak0gvKMfWeou9nczlQFoiIqK2YE9KG0T5eyDUxw0DInwaPHYyjyGFiIioLRhS2sEfborEYyOi4OKsQrwlsJwvqUJplWMvp0xERNQWDCntQO2kwl+n9MaxhRPw/RMJCPVxAwCc4topRERErcYxKe1I7WTOfDFBXjhfUoWDWZfw39QLSDpVgE4eGtwzKBzHc0vhoXHG/03sCUmS7NxiIiIi5WJIuQYGd/HD5pMFWLT2pLwtp7gKqTkl8v24MB9M7tdxV90lIiK61ni65xp45OYojOjuDwCQJGDRXf0wb0IMfN3VCPcznwp6c30aao0mAEBOcSXWHc1rdv0ZIiIiR8OelGvAxVmFD/8wEO9uzkB8hA/G9wkCAPxxdDeU62sx6o0knCmswKc7MvH4yK545It9OJVfjs8eHIRbezZczdbqYpke//jlKKb2D8XEvkHX6+MQERHZBXtSrhEPjTP+b1JPOaAAgCRJ8HJVY/7EGADAmxvS8PnOsziVXw4AWHc0r9nX/NO3B7H2aB6e/PrAtWs4ERGRQjCk2ME9g8IxoU8gDEaBl389Lm/ffLIAJlPdKZ+icj3WHsnFuUuVOH2xHLvPFMuPVdbUXtc2ExERXW883WMHkiThX3fHIqtot7zom9pJQmF5DVLPlaB3sBavr0vDZzszAQBxYd7oHeJt8xpHz+swJMrvurediIjoemFIsRMfdxeseOwmzFmZgkCtK6oNRvx6OBebTuTj/c0Z2HSyQN730LlS5Ov0AMxhxmAUOJRTwpBCREQ3NJ7usSM/Dxd89chQvDk9DuN6mwfMrjp4Xg4onz04CBF+7gCAPF01AOCh4VEAgEPnSq5/g4mIiK4jhhSFGN0jAE4qCRdKzWGkf7gPbu0ZiIGRvvI+3QM85anNh8+V2qWdRERE1wtDikJ4u6sxqF4gsfasxNe7eOGgLr6IDfWBJAHZxZU4fK4Eq1LO4WKZ/no3l4iI6JpjSFEQazABgLG9LCGlXnAZGOkHb3e1vFLtPcuS8ZfvDuGZ71Ja/F4fbzuDj7adbmOLiYiIrh2GFAWZ0CcIbmon9AnRokegJwAgJtALfh4ucFJJGGoZKPuXsd0hSUC1wbxi7c6MIhzIKsa+s8UY+foWrEo51+z7FJXr8eqaE3htzUlcKKm6th+KiIiolTi7R0HC/dyx6dlR8HBxli8+6OykwlePDIGuqhbhlkG00QFemDkkAt/vz0GPQC8cu6DDfzZloNpgRHZxJf7y3SFE+Xuif7gPAKBAVw0fdxe4OJsz6ZnCCvk9D+WUYGxP/+v7QYmIiK4CQ4rChPi4NdjW57I1UgDgn9P64oXJvVBYrsetb23F1lMXoap3UeXHvtyP1++OxaqU8/jvoQsY3zsQH80aBADIvFgXUlIZUoiISKF4uqeDkiQJHhpnRHbywNS4EACASQDB3q6ICfTCxTI9Hlq+D/89dAEAsKnearanC8vl10nNKUFZtQHfn1Hhkx1nr/vnICIiagpDyg3gj7dEw3J2CL8fHIFvHhuKmEAvqKS6wbhGk8DxXB0Sj+fjdEFdT8rhc6X43bK92Jmvwr/Wn0JOcWWL3ttkEqg2GNvtsxAREVnxdM8NIDrAE4+P6IoNx/Nxz+Aw+Htq8L8/3YyqGiO83dWY8p/tOHZBh4eX70PBZdOVqwxGmzEqT359AKVVBrx4ex+b2UYAYDCaUGsUcHNxQlZRBbaduojPdp7F+ZIq/PzUMPQNbXhaCgC2nbqIV349jnfu7d/kPkRE1D5KqwzYc6YIY3oFwqn+OAAA3+/LwYXSKtzaMwCxYT4AgA+3nka1wYiZQyIQoHW1Q4ubxp6UG8SCyb2w5bnRCPY2j2lxcVbB210NAOgZpAWABgElOsA8g2hAuDemRZp7Q45d0OHcpSq8tzkdr/52HPd8mIxyvflihnd+sBMjXt+C7ekXMfrNJPz9l2PILKxATa0JPx1sekbRt3uzkV5Qjm/3ZrfvhyYickC1RhN2pBdCX9t4L/ZbG9Lw+FcH8N7mDJvtJ3J1eP6nw/j3xnRMfX8nNp/MR7XBiGVbT+PfG9MVuUgoQ4oD6BXs1ej2zx8cjLfvicNXDw/G8EBh81h6QTk+3ZGJvWeLkZRWgOKKGhw9r0NhuR6f7siEEEC4n5u82NzmkwUQQjTyLkBGgXkMzNELuvb7UEREDurHA+fwh0/34OHl+3C+pAp7M4ttHj+QdQkAsHxXJqpq6oLM6tTz8t+FAP756wn899AFXKo0IMTbFbf0DLg+H6AFGFIcQK9gbaPbw/3ccVd8GDTOKrg4AW/e3Re3xZoXiqusMcIyzhY7Mwpx+mLdYNsd6YUAgJlDIvHlI0Ph4qRCVlGlzWkjK4PRhEzL9pO5OtQaTVfd7vr/uYiIbmRZRRUoqzY0+lhppQFvbUjDiVzzL3rrj+UBMK+RNf7trbhnWTJ2nykCYO5lSbf8Ynip0oAfLb3cJpPAf1PNEynenB4Hf08XnCmswPM/HgYAzBgS0eDUkBIwpDiAnkF1PSlP3xINH3c1/jK2R4P9pvYPwXsz4+VF46x2ZhThdEFdSKm1pJeYIE94apwxtKt5/y92nW0wiDarqFLeX19rwo6MQpxtJMxc7n+HLqDPi+vw44HmF6YjIuqIKvS10NcaYTIJ/HvjKYx6IwkPL9/XYD99rRGPfbkf727OwD9+OQrAPJZQfh3LL3MfbjWvIH62qBI1tXW/DH66/QyMJoE9mcXILa2Gl6szbosNxguTe8n7OKkk3Ds4/Jp8zrbiwFkH0MlTg6FRfrhQWoUnRnXFs+N7yIvFNWZQF1/sqdd9mF1ciaS0iw326xFoDj9T+gVje3ohvkzOwo70Qnz5yBCE+ZoXnsuoF24A4MHP90HtJOHLh4cioVunJtuw9dRFmASQfLoIvxsY1qLPS0R0rZVVG1BSaZAX2bwaR86VYkdGIaYPCsOYt7YiwEuDIVF++GaPebzevrOXkFtaJY8tFEJgwU9HsPes+ft4f9YlFJbrcbawbhbm6JjO2HbqIpLSLmLrqYsorzaPIYwJ9EKerhpniyqReDwf/z1kPtVzW2wIXNVOuCs+DF07e+LbPdmIC/dR3IBZK4YUB7Hy8ZtgEriq7rxBkX4ATkPjrEL3QE8cPa/DOkv3opWHixNCLQvP3Ts4HPpaE97fkoEzhRW458Nk/PL0zejspbE5TWRlMAo89c0B/PbnEfJrXM56iqigrLqFn5SI6NoQQkBXXQutqzOe+vog9mQW4ZfZN6N3SOOn1C/3zHcpOH2xAgeyLqG0yoDSKoN8asZqy8mLuHtgKD7edgbHc3VYcyQPTioJ/p4uyNfp8euhC8jTmb8XU/8xDj7uLpj9zUH8diQXD3y2V36d/uE+8PdywftbTuOFVUdQXFEDAHhwWBebfawrkysVQ4qDkCQJTld5uvHm7v64Z1AY+oV6Q1ddi6PnGw547R7oJffGSJKEB4Z1wfg+gZj58R5kFlbgX+tO4s3pcXJPSpS/hxw8unb2wJmLFfj5wDmUVBkgAfjrlF42vTvWffNK60KKEKLZHiAiomvl691ZeGN9GkqrDHhsRBR2ni6EEMB3+7KxcGrfJp93+FwJPthyGoO6+OK0ZbXvjSfybfZ5clQ3eGqc8OaGU9h8Mh/pBWX4fOdZ+fFXpvZFcYUeb244hY+3ZwIAfNzV8HF3MT8+rS80zir899CFeqfjvXBbXDA+2Z4pB5Sbo/0RE9T4RAql4pgUakDtpMLrv4vD/QldMH2Q7akWH8u05pjAhgd6sLcb3r4nDoB59PnH285gf5a5m/LpW6IxtlcgFt3VD/cNjQQArEo5j093ZOKTHZnYeKJAfp2Syhr5P1W+5TeGX1LPY+A/N+LznZk2s4iMpsZnFBERtRchBD7YkoHSKvPA1o+3m2c4AsD/DufC0MSEgP8duoDfLU3GumN5+OdvJxo8/vjIrlh8Vz88N76HPLNm66mLckC5/6ZILLt/IGYOjcD4PkEAgPOWi8J26eQhv46fhwvevrc/nhnbXd7WM8gLAV6u+OqRoegZ5AUXJxWevjW6bYWwA/akULMCvFwRHeAp94gMj/bHb4dz0Se08e7NARG+uHdQOL7bn4NX15j/U6ok4KZunXC3ZWzJAUtwqT8b6I31J3FrzwA4qSS5FwUAdNW1qKoxYsWebBRX1GDh/46jQl+Lp2/tjgNZl3D/p3vw1Khu+NOY7iAiOnq+FJU1Rgy5bALA1aqpNaFCXwuNU90vQGeLKnHB0qurkoD6vxsVV9Rg4/F8ODupUFyhxz2DwiFJEi6UVGHBz0dQYzRB46yCvrZhkHl4eBSCvM1jQXoHa9HV30P+XpyVEImX6/XQdA/wRN9QrdyzHeXv0eD1nhjVDTsyCnG+pApxltM4Q6L8sHbOCFQbTHBzcWpVTeyJIYWuaNFd/fD7j3ZjbK8ALJjUEwPCffD7wRFN7v/ytD6I6uyBzScLEKR1xaMjomzGnvQJ8YazSpK7JQHgVH45Ji/ZjsJyPYosvShW+bpqm/O2n+7IxFOjo/G7D3dBCOCtxFMMKUQOKqe4EoXlegyI8EWBrhrTP0xGjdGEpOdGX/Wg1nVH85CeX4aBXXzx529TUFheg2BvV/zJ8rWy67R52YWhUX6I7OSO7/ebZx32DtbieK4Oc1amosbSm+LvaR4MO+/HQyjX1yI+wgd/GdcD93+6F04qCd0DPHEyrwzhfm5yQAHMp81/fGoYDmRdQq3R1GDFb0mSMCuhizxluLHPpnZSYcWjN0GSYHNqXJKkDhlQAIYUugqDu/gh6bnR8PNwgYfGGY+O6Nrs/hpnJzw5qhueHNWt0cdd1U7oFazFkfPm1Q0fHh6F7/fnIC2/rNH9U3NK5NM/Ls4qXKo04Lt9Oai/dhzHqxA5HiEEHvhsL7KKK7H+mRH4Zk+2PD038Xg+Hr45qtHnfb4zEz8dPIdwX3dM7R+Kp1cctPmlCQByS6vxv2wV4vLLsOGYeQzJsG7+GNTFF9/vPwcXJxWWPzwYf/kuFTsziuTn/fnbFPh6uODcpSq4OKvwr7tj0T3QC/++tz/cXZyQVVSJV9ecwM3RnRu0y8/DpUE4qe+OuBA5pAQ1MRtHpcC1TtqCIYWuSkum2V2NuHBvHDlfCje1E/5vUk88Maor1h7JxQ8HzuHYZSvTWgeZ9QzyQpS/B9YezcMLq47Y7FNSaYCvh0u7tpGIGpdTXAlPjXOL/s+du1QJjbMTOntpmtxn0doTuFimx+t3x8LZqfEhk8u2nsbO00V4b+YAlFfXyqdHvtuXI0/lBczfGw/fHIWNx83fH2N6BUCSJFQbjHhjfRoqa4w4el6HtUdtZy72CPTEgkm98NDyfUguUGHKe8nyY8OjO2FgpC+eG98DYb7uCPByxecPDsE3e7Lg7abG3O8PoaLGiIqaKoT5uuHf9/ZHd8v4vWkDQgGYF1sL0GowukfLV3d1VTvho/sHYsPxfNwVH9ri53dEDClkFwld/fH17mwMj+4EF2cVArWueHB4FCb3C8aQ1zYBAFzVKlQbTHJI6RfqjYGRvvKXSv1TRmuO5qJAp8e43oHyRQxzS6uwZGM6jpwvxW2xIXhqdOM9O0R09XJLqzDuna3oGaTF6tnDr+o5heV6TPz3dni7qbH5uVHQODc89ZBZWIFlW88AAGYOicCgLn44fbEcEoAQHzcs33UW5dW1eG+L+Xo0vx3OhbebWn7+pzsyYRJ1Mwn3ZBbj/S0ZeGN9GgBgUt8gDI3yg5uLEyotC6D1CdHi2AUdJAl443dxOJGrw8M3m09PTx8Yih8OnJe/h0J93BAb5gNJkvD0rXWnl12cVXhouLnHJjWnBF8mZ2FEd3+8NzPepn1Wzk4qTO3f+oAxvk+QPIjWETCkkF1M7heED/8wEPGRPjbbA7SueON3sUg+UwStqxrLd51FtcF8rrdfmDdGxdR1kT59azSSTxdhT2Yx/rrKvBLjkk3peGVqH9wVH4YHPtuLU/nmsSyZhRV4bERUg9/OjCaBjSfyERfi2Wx7fzuci+gAzw43fY+ovR3KKUW1wYTUnBLkllYhNbsEY3sHQl3v/1a1wQiNs0o+Bbs17SLK9bUo19ci8Xg+bosNafC61iXbAfMq18UVNXjy6wPw1DjjiVHd5LBhlZRWgIh6PbzWszV/HhOND7acRnpBuc1z1h7Ns+k1efTmKDwxqhvm/3QYAyN9Gywa+codvdFbZGHm1EkoqKiFl6szXJybnxD74u19cM+gcPQK1ipyifmOiCGF7EKSJEzs2/hvA9MHhWP6oHB8sv2Mzfa+od4I9nbDn8d0x8UyPWbfEo0LJVU2q+MCwIdbz2B/1iWcyi+Hv6cGheV6VNYYcTxXh36h3jZjV1767zF8tTsL8RE+mGX53jSZBD7cdhrhvu64PS4EyaeLMHvFQUgSsP+vY9HJs+nuaqKO6PgFHUoqazAs2v+K+9affffw8v04kavDn26NxrPjYwCYB6H+eWUK7hkUhpggLX7Yn2Pz/JV7c2xCyoGsYnyZnIVf6oWU97dkQEDAJMwz/FbuqzuNY+0p2ZlRhLwA2yu7e7k6Y1LfYEiQ8NqaE6jQ1+L+hC6Y0CcQ/z10Af9NvSAPzJ8cG4zOXhp89uDgRj+nk0qCn8Y8xuNqT3c7qSS5J5faB0MKKVZgvYFhsWHeiAvzAQDMHVd33aHIemsFdO3sgQKdHudLqnA+tQqSBCy7Px7vbzmNzScL8KdvU5BTXInYMB88MbIr8nTV+Gp3FgDgYHYJ+rtKmALzb1yvr0uDs0pCfKQv1h7NBWC+auhra05i/qQYPPT5Pgzr1gnzJ/Zs9Nx5rdGEXw/nIrKTOwZE+Lb4swshkJpTgr6h3ja/oRK1N5NJYNZne1BYXoP3Z8ZjiuUio005U28VaesF737Yfw7PjO2Bw+dKMGdlCmpqTVixJxsuziq5J9RqR0YhsooqEO7rjl+P5GLeD4fk6bnW6b01l607klNsXhvky4eH4OZofwx5bSMKy2twKKcEgHnl1NScEtw5IBSuaidMGxAqjwGxGhDhi2n9QzHj490I83VDf8v3CSkbQwopVphv3bTlN34X12j3af0Fjcb2CkRhuR4/H6y7RsXASD8M7nIJm08WIKvIfL2L1JwSPPXNQfl5MYFeSMsvw6/ZKsw3Cby7OR2A+UKKH209jU31Fpr76eA5hPi44tgFHY5d0CGrqBLL7h8ISZIghMDitSfx30MX4OJsvjK0q1qF9c+MlMPUuqN5KCirxqyELs1+9q92Z+EfvxzDEyO7YkG9C4FRx2E0CagumwranGqDER8knYaTJGHO2CtPqdfXGvH31UcxuIsf7hwQioyL5YiptxL0lfzv0AX887fjePqWaBSWm3sX5v90GL2CvRDuY+4tPJVfhnk/HcMTo7rK4ygau9p5nq4a3+3LwVsb0qCvNcnjxS4PKCO6+2N7eiEWrz2J7OJKeZB8/3AflFYZcEdcCJZsSpf3/8NNEfh6d10vSp8QLVQqCSO7d8bPKeb/51pXZyy7fyB+OngO998U2exnjgv3wY75t0LjrLrhZsHcqFr8K9q2bdtw++23IyQkBJIkYfXq1Vd8TlJSEuLj46HRaBAdHY3ly5e3oqnkaPqH+2DehBh89uCgJseCRHaq64a9Odpf/iKVJODPltUVB3ep68nw0jjj0XrTEv98azR+fCoBaicJBdUSvtidjZN5ZVBbriHwRXIWzpdUwVWtQrfO5qDxv0N13dIbjufL414+3ZGJZdvOILe0Wg5E1QYTFvx8BEIIZBdV4ukVB/GPX47hqGX6NWCeKfHFrrOorffb47d7zV3kPx08Z7OdOoZqgxHj39mKaR/sslkhuSnl+lrcvXQX/rMpHe9sPIXsosorPmf9sXx8v/8c5v14GHd/mIyJ/96OVZYf3I1Jyb6EyUu2Y+PxfOSVVuNP36YgX6fH3385ZtOOP35zEFWWgaWvr0/H8Vwd3k48JX+OzCauYv7CqiMoqqhBnxAtPp41SN5uXVTs9rgQzLGsZ7T2aB6OXdDBU+OM2bd0ww9PJmDLc6Pxl3E98LBlEOqrd/a1mQET4u0qn2p9fFRX3NTVD8OjO+GVaX0RqHXFH0dHw8u14UDVy1mXUqCOocX/UhUVFYiLi8PDDz+Mu+6664r7Z2ZmYsqUKXjyySfxzTffYNOmTXj00UcRHByMCRMmtKrR5BgkScLsW5pfxrn+qouDu/hB46zCU6O7IcTHTZ761y+s7hzx/03uifuGRmJc70BU15owqod5IG6fEC1Sc0rxfpL5cuezErrg+AUdks+Y1z8Y0b0zPDXOOH2xAmcv+wGy+0wRtG7OWLT2JADzJQC6B3oizNcNMz/eg12ni5B8pgg/Hzwvz0bamVGIvqHeMBhNGPH6FgCAh8YZvxsYhoyCMrkbvbC8BrvPFOPm7rZjBQ7llOCL5LNYMKlXs1M6L5evq8bvPtyFCb2D8Lfbel/18xpz9HwpyvW1uKlr01ezdiTnS6pQazQhspMH9p0tlq/TUlhe0+i/0YlcHdYeyYW3uwu8NM42U++3pV/EiW06/G5gGLKKKrHheB5eu7OffK0WwHyFcCvraY8fD5xDj0AvGIzm1UX/uuooHhreBZP7BuNvq4/ieK4O83863OgF8Sb2CcL+rEs4mVeGV9eeRLge2JpuXsQsq6gSh8+VIrKTu7xmkdVdA0Lla8b0DPLCZw8ORoCXBvMmxMBkEnhqdDesOZqH4d06oZOnBsOjO2FnRhG0rs5YPXs4una2HbQ+f1IMZg6NQHSAp3xZDADoU2+sR88gLVY+nnDFfxPq+FocUiZNmoRJkyZd9f4ffvghoqKi8NZbbwEAevXqhR07duCdd95hSKE289A4Y+PcUXBS1a2oOH9iT5t9NM5OePueOJy+WI57B4UDAIZe9oO1f5g3UnNKUVplvsz56JjOeG58DD7flYnNJwrw9C3R2GlZddLK+mW7J7MIlTVGGE0Cg7v44tnxPeQu9zsHhGLlvhwsTTqNnRl1z991ughPjOqGlfvqBhXuSL8IZ5WE7y8baLhs22lU1NQioVsnaF3VMJoEpr6/0/z5XZzxyrTGL25mMgnszixCfIQvXNXm2vx6OBc5xVVYuS8HCyb3avQUWq3RhPk/p6KzVoMFkxo/1ZRTXInb3t0BJ5WErfNGI8y38YGFJpOAwNVdfbs9mUwCBpOp0amuVgajCRkF5lMkLe36L6msgbtL3WyPqhojpr63A1U1RmyZNxoHs0rkfbOLKxqElJTsS7hnWTIMRnNojQuzHWz5z9+Oo9pgQvKZIhSW6aGrrkW3zp7y4FQASL7seASA/Wcv4e6lu2AwmhCodUVuaTUKy/UQAnIIKqqowfb0QkgSbBZEvLVnAGYNi8TMj/fgu/3n0dnVXDvrfv89dAGT+5nHqwR7uyLc1x2Hz5dgztjueG5CDFSShECtRj726/+CcUdc3UDZhXf0xevrTuKJUV0bBBTA/P81OsC8PVDrikCtBvk6PfqGcECqI7rmI/KSk5MxduxYm20TJkxAcnJyE88gapnoAM9Gr2NR313xYZg3ofFBrgBsLlfu4qzC4C7m9RT+ODoaPz41DHHhPugRYHvKyXqhxD1nivHTQfMy2b8bGGYzJmCS5Ut9e3ohTALyl+++s8VIyb6EN+tNkdyTWYxnvkvFLstvyL8fHC4/94mvDmDwPzdiy8kCm9NNR+qdNgLMP5ytlmxKx8yP9+CdxFPytu3pFwGYu/XPXCxHVlEF/vDJHvxsaT8AJJ8pxs8p57Fs6xnkFFdi2ynz9NH6Fv7vOADzuIttpxr+sLSa+30q+r+8AecuVUIIgX+tO4nHv9yPmkauY9IStUYTUrIvyacgPt+ZidiX1sun0V757Th6/n0d/vxtCi5d9pu/1b83nsKkJdtx70fJyClu/PSK0STwn03pGL54M/70bQoAYFdGIYa+tgmPfLFP3m/D8TwUltegosaIHw+cw96zdb0cWUWV0FUbcP+ne7DIcj2r9zZnyAEFAA6dM7d7jOUic9axHGcuVkBXba79u5sz8LfVR/DNHvMpyLNFlXBSSUhecCv2vjAGkZ3cUWM0QV9rgkmYV0y1vv/fVpun6I+w9MhpnFX44qEh8n0A6BOqxbBu/piVYD6uL1ZL8HZzxoJJ5tD/S+oFHDlXAsA8SP3TBwdhy3OjEdnJAyE+5iXer2Y8THSAJz6aNQgDI6/u2joT+gTBSSVhTK+WL35GHd81PzGXl5eHwEDbZX4DAwOh0+lQVVUFNze3Bs/R6/XQ6+umlul05t8ADAYDDAZDu7XN+lrt+ZodEesA9AmqCznx4d5wggmGywb9RXWqm23k5eqMkdF+cFWrUFRRg6KKGmicVRjX09+mjoMjtPBxU6PEcvXUN+/ui4e+OIBLlQbc+cEuAICHxgkVeqP8QwUAFkzsgQcSIhHh54pj58tw9IIOWcWVWPi/uvEDAJBeUIZqfQ2cVBK2ZxRi7vdHcHtsEOaMicanO8yXdP/tSC6eHdsNNUaB3WfqfnhuP1WAf2/KgK66FnsyizAhZhQAYM2RXHmfGR/vxrlLVejq74GP7x+ACD93HMi6ZHOp+W2nCjA9vuGMkL1ni7HaMq30t0PnUaE3YqnldNru0wVIaMNpopd/PYGv9uTghUkxeGhYJL7bmw1ddS1WHcxBd383/LD/nPzbv8ZZwmvT+jR4Deu6HPvOXsKclSn47rEhOFtUged/PIK+rhLGGQz4dnc23raEvPMlVbgzLghPfJMCg1Fge3ohUs4WISWnRL6WCwB8nZxlc/2pzItleOmXi9ieXojt6YUY09Mfm04WQJKAeweFYeW+uufOHBKGTSfrBmpfzjqINDbMfLqmb4gW/u7mr/FbevhjebL58WFd/XDoXClCfdxwqqAcpVUGBHhp8J97Y3Eyrwz+ni7o0skDh7J9sD29EGonCVF+rjAYDPjLmG7Ym1mEopIyfPJAPLoFavFlchbOXarCK5Yr+cYEeMLVCXB1d77m3xv/N6E7/jS6K3zc1df1O4rfi2ZtrUNb6yeJqxnV1dSTJQmrVq3CtGnTmtynR48eeOihh7BgwQJ525o1azBlyhRUVlY2GlJeeuklLFy4sMH2FStWwN29fZdnJwLM3dn/OOAEnUHClHAjxoc1/G9hEsDze51gMEmI9BSY28+IpcdVOFlq7p0Z0tmE+6Ib9hB8e1qF3QUqxPqZ8EiMCctPqZBSZH5Od60JD8eY8OIBJ9SYzL+FTg43YsJl719tBBYecEKl0byPp7NAea357/PjauGnARalOqGkxrytk0agSF/3W+0L/WtRUiPhg+NNn/5YEFeLzm7A3/c7oaK24W/E4R4Cz8Ua8d0ZFXblqxDgKlBQLcHdSeDVwUbUP2MiBPCfY044U2beGOoucL6yboe7uxgxMrh1Xz0leuDlFCcYhQQ/jcDzsUYs2OcEAQlRXgL3dDXiX4fqfv/ydxX4W38jjAKwrsVVWA28kmL7O9pL8bV475gTCi11W5JQiw9PqHCipK73Ta0SMJjqPodaEjAIqcnHASDAVeBiNSBg3h7sLpBbKaG/nwljQk1464i5He7OAq8OMuKFfU6oMpo/W7FeggoC48IE1p9TwddF4FJN3etPDDNhUrj5mDtXAbx9xAnx/gJ/iDah1gScKpWw7KT53/z+aCMGdb7s+jSVwBuHnRDjLfBEr7pj1yQACeZTPQCQXirhPcux00kj8ExfI7S8CgVdhcrKSsycOROlpaXQahuOhbqSa96TEhQUhPz8fJtt+fn50Gq1jQYUAFiwYAHmzp0r39fpdAgPD8f48eNb9SGbYjAYkJiYiHHjxkGtvvKo8BsV62CuwYbzG3G4zAPPTR9sM/25vk+yk3HsQhniu4di8uS+6DWkAmuO5qN7gAdG9egMTSMrUg6tqMH3+8/h3kFh8PNwwYDh1fjvoVzEhXljcBdfOKkkfJe3B6k55i7/ByclYEC9009W5z1O4z9bzD0R7943CB9uO4M9mZeg7RKLw7llKKnJga+7udfGGlC8XJ1RVl2LRYecLW0zwc9DjeKKut9u1E4SDEYBY0AM0nJOoaJWgptahSpLT1K4rxtyLlUhp0LCsNHj8GLqDgAGLLpnIOZ8dxjl+lpExA1Hv1Ct3N2/4Xg+zuw+VNf2Stsf3JqASEyebDtwN6OgHBF+7tDXmpCWX4b+Yd6oNQmYhIC7S91X1cu/noBRmMftFOslnNZEQ+AsAOBClROcQ3sCh9LQJ8QLxy6UobBawqqiIGzPKMTgLr545Y7e5lNqKScxpIsvTEJgf1YJtpQHoVBfd+qqe/xwZB/cD8CIGYPD8O2+czCYJKgk4PeDw7Bi7zmbgDI0yhfx4T5Yus3cg2WdhltQbfvZcy21eHJSPEZ198eHp7agQm9EfBd/3DZlIH4pPoikU4X487he8HJVw8PFCWN6BeBsUQVCfdzwv8O52He2BHFh3pjWP1gebwQA991RCze1kzzGptZoQu7Px+Dp6oS/39ar0dMxk8ZWQeumhme9GS+NfSdod2VhS9pFvHJHb5tZdTcqfi+atbUO1jMhrXXNQ0pCQgLWrFljsy0xMREJCU2PzNZoNNBoGo6GV6vV1+RguVav29E4eh0mhAksmTyy2Rr0C/XBsQtl6BPiDbVajR7BPugR7NPs6wb5qPHnsXUDHiP81Xh6jO34lj4h5oG7Xq7OiI/s1OjYmcdGdUNGYQWGdPHDLb2CsC+rBHsyL+GTnVnytND/zBgAX3cXHD5XCh93Nc5fqsKra05ACPM4h8hO7nhhci888dUBAObrmQzu4oeXfz2OdScuIqfA/L7TB4Vjb2YxTuaV4dnxMVi89iTydNVYvjsbJVUG+HtqMLpnEIZHn8f6Y/lYsuUMLpbpoXFW4atHhmDROvMpkoeGd8EXu87KS5aP6tEZW09dxMHsUjz5TSq6dvbA3HExSM0pwYyPd2Nkj84orzbgYHYJ/D1dUFplgMEoEO7nhpWPJ2DDsTx8tcccUOLCvHHoXCk+3nFWrpG+1oSvLY+P7x2MyhoTMgsr5FkqezIvYf6qY/C1zJIZ3TMALk4q7M8qwdbLxtZ8lnwOFXojvFyd8ZdxMVhpOYU0tX8onr61B1bsNZ+meWJkV4ztHYjuAZ7wcXfBPUMikZ5fBg+NM+77ZI/8etbPDgAuTiqMjAmEm4szhnTxw5a0i4gL94FarcYr0/phW/pF3Dso3OY46B5kPs7uHdIF9w5p/FjzuezYVauB/8yMb3xni8jOTR/v9b8THh8VjcdHNT/b7kbk6N+LVq2tQ1tr1+KQUl5ejoyMDPl+ZmYmUlNT4efnh4iICCxYsADnz5/Hl19+CQB48skn8d577+H555/Hww8/jM2bN+P777/Hb7/91qaGE9nDX8b1QO8QLe6KD7vyzi0wtGsnfLMnG7f2DGhycK+Xqxof3DdQvj882h8fJJ2WA8pTo7thRHfzlGrr0tyZhRVYvO4k3NVOeOueOPky8P1CvXGhpAp/v603qg1GvPwrcPicDoCESD93PDchBoVlepzILcPkfkH4JfU88nTV+MwSCG6LDYaTSsKcMT2w+WQBtll++ALA/Z/uxblLVQj2dsW8CTHYc6YYx3N1cHFS4clR3bD11EWczCvDybwybDppnuk0IMIHAGxex7rAGGBecfThz/chLb8MAPDEqK64d1A4Jvx7m80AVADyGjVDovyQWVgu18fbTQ19rREp2SXyvrfEBMBT44x/WsZaDIz0xZBIc2/Ij5ZFAYd08UOA1hV3xIVge3oh5ozpjhAfN9w3NAJpeWX44y3RNheSi/L3QJS/B3TVdb1VkgT5s5v/vf3k3qHnJsTA18MFDw4zrw8S7ucuD8omcnQtDin79+/HLbfcIt+3npZ54IEHsHz5cuTm5iI7u951FqKi8Ntvv+Evf/kLlixZgrCwMHzyySecfkwdUqDW9YqrxbbG7bHB0Lo6t2gJ/eHR/vjqkSH4bl8OtG5qPFvvcgFWUf4e+PVPN6OThwsC6l1m4KenhkFfa4SXqxpCCDw2IgqbThSgvLwc782Ig9ZVDa2rWp4i2itYiy1pF1FlMC/yNcFyFdbeIVrMHReDf607KZ/eSLWs2fHC5F5wd3HGwEhfHM/VYWQPf/QP92kw9fXYBR0ulFTZtPu58T0QG+aDyE7uKKk0YNoHO+WA8ujNUfi/iT0hSRLmjOmONzeYe21uiemMLWnmEODrrsaACB+czNPJg3efGWu+5tMHlsG7T4zsil7BWvmxfF01/n5bb6TnlcqnbABzoACAJb8fAKNJyNOpX72zX7P/PlpX2+AyIMJHPrVmXZ8HMPeivX1P/2Zfi8hRtTikjB49utkVFBtbTXb06NFISUlp6VsROQxJkjA6puVTLEd07yz3njTF+oO4PhdnlbzGhyRJ+OuU3nh+fHesWbMGPRtZ3bf+4l9aV2cMqreK75OjumJgpC8CvDSYuGQbqg0mDInyw22Wa8A8PrIrSqoM+NOt0XBzcYIEwPoNMqSLH/aeLcalSnOvg5fGGQMiffH4yG5y+yI7AXfHh+HHA+cwNMoP/zeppzy24slR3XD4XCkqamrx99t6I+fSAUT5e+AvY3vAVe0kr3YqScCUfsFwdXHC8VwdegdrMW9C3Sm4Z8bWBbxeQV64OdCEszUecFZJNhfDa+l6L9YLXM4cEgFXtRMm9AnCjoxCeb0RImoe1wYmoiuqH3RGxwTYXPRQkiQMiTL3Nswd1wMr9+Xgn9P6ykEi3M8d784YIO9fbykXTIkNxt6z5qtYO6kk7PvbWJuBoFavTO2LoVF+mNA3yOZ0mLOTCh/VW4J949xRNs/rH+aDB4d1QaiPm9yTtPyhJgZ01Ps807uaMHnyiDafT1/5+FAcyLqEeyyLCL47YwBqTYIXjSS6SgwpRHRFXTp5wFVtvqJtc4tqPT6yGx4f2a3Z1/rX3f3wfz8fwYd/GIgIv7pZIt0DPBsNKADg5uKE6ZYf9C2hUkl46Y6Ga6RcL9EBXoiutwigJEnydaGI6MoYUojoiqyDZFNzLmF876A2vda9gyNw54AwuDirYDIJeLupUVplQB8ue05El2FIIaKr8tTo5ntIWsI63kSlkjCsWyesPZqH+Eifdnt9IroxMKQQkV29dEcfjOjeGfcMat9p3UTU8TGkEJFdBWpdMXNohL2bQUQKxCHmREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSAwpREREpEgd4irIQggAgE6na9fXNRgMqKyshE6ng1qtbtfX7khYB9YAYA2sWAfWAGANrNpaB+vPbevP8ZbqECGlrKwMABAeHm7nlhAREVFLlZWVwdvbu8XPk0Rr4811ZDKZcOHCBXh5eUGSpHZ7XZ1Oh/DwcOTk5ECr1bbb63Y0rANrALAGVqwDawCwBlZtrYMQAmVlZQgJCYFK1fIRJh2iJ0WlUiEsLOyavb5Wq3Xog9CKdWANANbAinVgDQDWwKotdWhND4oVB84SERGRIjGkEBERkSI5dEjRaDR48cUXodFo7N0Uu2IdWAOANbBiHVgDgDWwsncdOsTAWSIiInI8Dt2TQkRERMrFkEJERESKxJBCREREisSQQkRERIrk0CHl/fffR5cuXeDq6oqhQ4di79699m5SqyxatAiDBw+Gl5cXAgICMG3aNKSlpdnsM3r0aEiSZPPnySeftNknOzsbU6ZMgbu7OwICAjBv3jzU1tba7JOUlIT4+HhoNBpER0dj+fLl1/rjXbWXXnqpwWfs2bOn/Hh1dTVmz56NTp06wdPTE3fffTfy8/NtXqOj16BLly4NaiBJEmbPng3gxjwOtm3bhttvvx0hISGQJAmrV6+2eVwIgX/84x8IDg6Gm5sbxo4di/T0dJt9iouLcd9990Gr1cLHxwePPPIIysvLbfY5fPgwRowYAVdXV4SHh+P1119v0JYffvgBPXv2hKurK/r164c1a9a0++dtSnN1MBgMmD9/Pvr16wcPDw+EhIRg1qxZuHDhgs1rNHb8LF682GYfJdfhSsfCgw8+2ODzTZw40Wafjn4sXKkGjX0/SJKEN954Q95HUceBcFArV64ULi4u4rPPPhPHjh0Tjz32mPDx8RH5+fn2blqLTZgwQXz++efi6NGjIjU1VUyePFlERESI8vJyeZ9Ro0aJxx57TOTm5sp/SktL5cdra2tF3759xdixY0VKSopYs2aN8Pf3FwsWLJD3OXPmjHB3dxdz584Vx48fF++++65wcnIS69atu66ftykvvvii6NOnj81nvHjxovz4k08+KcLDw8WmTZvE/v37xU033SSGDRsmP34j1KCgoMDm8ycmJgoAYsuWLUKIG/M4WLNmjfjrX/8qfv75ZwFArFq1yubxxYsXC29vb7F69Wpx6NAhcccdd4ioqChRVVUl7zNx4kQRFxcndu/eLbZv3y6io6PFjBkz5MdLS0tFYGCguO+++8TRo0fFt99+K9zc3MSyZcvkfXbu3CmcnJzE66+/Lo4fPy7+9re/CbVaLY4cOXLNayBE83UoKSkRY8eOFd999504efKkSE5OFkOGDBEDBw60eY3IyEjx8ssv2xwf9b9HlF6HKx0LDzzwgJg4caLN5ysuLrbZp6MfC1eqQf3PnpubKz777DMhSZI4ffq0vI+SjgOHDSlDhgwRs2fPlu8bjUYREhIiFi1aZMdWtY+CggIBQGzdulXeNmrUKDFnzpwmn7NmzRqhUqlEXl6evG3p0qVCq9UKvV4vhBDi+eefF3369LF53r333ismTJjQvh+glV588UURFxfX6GMlJSVCrVaLH374Qd524sQJAUAkJycLIW6MGlxuzpw5olu3bsJkMgkhbvzj4PIvZZPJJIKCgsQbb7whbyspKREajUZ8++23Qgghjh8/LgCIffv2yfusXbtWSJIkzp8/L4QQ4oMPPhC+vr5yDYQQYv78+SImJka+f88994gpU6bYtGfo0KHiiSeeaNfPeDUa++F0ub179woAIisrS94WGRkp3nnnnSaf05Hq0FRImTp1apPPudGOhas5DqZOnSpuvfVWm21KOg4c8nRPTU0NDhw4gLFjx8rbVCoVxo4di+TkZDu2rH2UlpYCAPz8/Gy2f/PNN/D390ffvn2xYMECVFZWyo8lJyejX79+CAwMlLdNmDABOp0Ox44dk/epXzPrPkqqWXp6OkJCQtC1a1fcd999yM7OBgAcOHAABoPBpv09e/ZERESE3P4bpQZWNTU1+Prrr/Hwww/bXJjTEY4Dq8zMTOTl5dm019vbG0OHDrX5d/fx8cGgQYPkfcaOHQuVSoU9e/bI+4wcORIuLi7yPhMmTEBaWhouXbok79NR6gKYvyckSYKPj4/N9sWLF6NTp04YMGAA3njjDZtTfTdCHZKSkhAQEICYmBg89dRTKCoqkh9ztGMhPz8fv/32Gx555JEGjynlOOgQFxhsb4WFhTAajTZfxAAQGBiIkydP2qlV7cNkMuGZZ57B8OHD0bdvX3n7zJkzERkZiZCQEBw+fBjz589HWloafv75ZwBAXl5eo/WwPtbcPjqdDlVVVXBzc7uWH+2Khg4diuXLlyMmJga5ublYuHAhRowYgaNHjyIvLw8uLi4NvpADAwOv+PmsjzW3j1JqUN/q1atRUlKCBx98UN7mCMdBfdY2N9be+p8nICDA5nFnZ2f4+fnZ7BMVFdXgNayP+fr6NlkX62soSXV1NebPn48ZM2bYXDTuz3/+M+Lj4+Hn54ddu3ZhwYIFyM3Nxdtvvw2g49dh4sSJuOuuuxAVFYXTp0/jhRdewKRJk5CcnAwnJyeHOxa++OILeHl54a677rLZrqTjwCFDyo1s9uzZOHr0KHbs2GGz/fHHH5f/3q9fPwQHB2PMmDE4ffo0unXrdr2beU1MmjRJ/ntsbCyGDh2KyMhIfP/994r6wXm9fPrpp5g0aRJCQkLkbY5wHFDzDAYD7rnnHgghsHTpUpvH5s6dK/89NjYWLi4ueOKJJ7Bo0aIbYnn43//+9/Lf+/Xrh9jYWHTr1g1JSUkYM2aMHVtmH5999hnuu+8+uLq62mxX0nHgkKd7/P394eTk1GBmR35+PoKCguzUqrZ7+umn8euvv2LLli0ICwtrdt+hQ4cCADIyMgAAQUFBjdbD+lhz+2i1WkWGAB8fH/To0QMZGRkICgpCTU0NSkpKbPap/29+I9UgKysLGzduxKOPPtrsfjf6cWBtc3P/14OCglBQUGDzeG1tLYqLi9vl2FDSd4o1oGRlZSExMdGmF6UxQ4cORW1tLc6ePQvgxqmDVdeuXeHv729z/DvKsbB9+3akpaVd8TsCsO9x4JAhxcXFBQMHDsSmTZvkbSaTCZs2bUJCQoIdW9Y6Qgg8/fTTWLVqFTZv3tygG64xqampAIDg4GAAQEJCAo4cOWLzH9T6Jda7d295n/o1s+6j1JqVl5fj9OnTCA4OxsCBA6FWq23an5aWhuzsbLn9N1INPv/8cwQEBGDKlCnN7nejHwdRUVEICgqyaa9Op8OePXts/t1LSkpw4MABeZ/NmzfDZDLJIS4hIQHbtm2DwWCQ90lMTERMTAx8fX3lfZRcF2tASU9Px8aNG9GpU6crPic1NRUqlUo+BXIj1KG+c+fOoaioyOb4d4RjATD3tA4cOBBxcXFX3Neux0GLhtneQFauXCk0Go1Yvny5OH78uHj88ceFj4+PzayGjuKpp54S3t7eIikpyWbKWGVlpRBCiIyMDPHyyy+L/fv3i8zMTPHLL7+Irl27ipEjR8qvYZ16On78eJGamirWrVsnOnfu3OjU03nz5okTJ06I999/X1HTb5999lmRlJQkMjMzxc6dO8XYsWOFv7+/KCgoEEKYpyBHRESIzZs3i/3794uEhASRkJAgP/9GqIEQ5plqERERYv78+Tbbb9TjoKysTKSkpIiUlBQBQLz99tsiJSVFnrWyePFi4ePjI3755Rdx+PBhMXXq1EanIA8YMEDs2bNH7NixQ3Tv3t1m2mlJSYkIDAwU999/vzh69KhYuXKlcHd3bzDl0tnZWbz55pvixIkT4sUXX7yuU5Cbq0NNTY244447RFhYmEhNTbX5nrDO0Ni1a5d45513RGpqqjh9+rT4+uuvRefOncWsWbM6TB2aq0FZWZl47rnnRHJyssjMzBQbN24U8fHxonv37qK6ulp+jY5+LFzp/4MQ5inE7u7uYunSpQ2er7TjwGFDihBCvPvuuyIiIkK4uLiIIUOGiN27d9u7Sa0CoNE/n3/+uRBCiOzsbDFy5Ejh5+cnNBqNiI6OFvPmzbNZH0MIIc6ePSsmTZok3NzchL+/v3j22WeFwWCw2WfLli2if//+wsXFRXTt2lV+DyW49957RXBwsHBxcRGhoaHi3nvvFRkZGfLjVVVV4o9//KPw9fUV7u7u4s477xS5ubk2r9HRayCEEOvXrxcARFpams32G/U42LJlS6PH/wMPPCCEME9D/vvf/y4CAwOFRqMRY8aMaVCboqIiMWPGDOHp6Sm0Wq146KGHRFlZmc0+hw4dEjfffLPQaDQiNDRULF68uEFbvv/+e9GjRw/h4uIi+vTpI3777bdr9rkv11wdMjMzm/yesK6hc+DAATF06FDh7e0tXF1dRa9evcRrr71m8wNcCGXXobkaVFZWivHjx4vOnTsLtVotIiMjxWOPPdbgF9OOfixc6f+DEEIsW7ZMuLm5iZKSkgbPV9pxIAkhRMv6XoiIiIiuPYcck0JERETKx5BCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIr0/weEfjZwOitTAAAAAElFTkSuQmCC"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 推理"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "    hidden = None\n",
    "    text_generated = []\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len))\n",
    "    print(start_string, end=\"\")\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            idx = torch.multinomial(probs, 1).item()\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \"\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-12T09:44:59.723306Z",
     "iopub.execute_input": "2023-12-12T09:44:59.723605Z",
     "iopub.status.idle": "2023-12-12T09:45:01.325354Z",
     "shell.execute_reply.started": "2023-12-12T09:44:59.723580Z",
     "shell.execute_reply": "2023-12-12T09:45:01.324258Z"
    },
    "trusted": true
   },
   "execution_count": 14,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/1000 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "44faaa03ad0540c892973b6c85b8fa65"
      }
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "All: I will affords.\n\nDUKE VINCENTIO:\nBut he, my lord,\nHe came all aloof,\nOur knees have set their master's hate.\n\nPOLIXENES:\nWhat may be so soundly.\n\nPedant:\nAy, and must be a man.\nBut did thee farewell.\n\nPERDITA:\nNo, my good lord!\n\nLADY CAPULET:\nWell, then, let me speak.\n\nQUEEN MARGARET:\nAy, sir; well, there is no more done,\nAnd there thy son should be thus.\n\nMENENIUS:\nThat's a meeting of a town,\nTo what may shed him to your highness\nWhich he comes to see thee here.\n\nBUCKINGHAM:\nThen she is only have sentenced there,\nRevoke the world.\n\nNurse:\nWhat is thy sword to the sacrament.\n\nRICHARD:\nWhat, will our tents, and so deal them now.\n\nServant:\nMy lord, you look on him.\n\nSLY:\nMarry, and more than time\nWith some shoulders.\n\nGLOUCESTER:\nWhy, here's the prettiest of your face.\n\nProvost:\nAll this time were sentence,\nShall marry her lip, he did indeed.\n\nQUEEN ELIZABETH:\nO that will stand to her stand,\nI'll make my head with bright hand and her beauty tribune.\n\nQUEEN ELIZABETH:\nThat is your good wo",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {},
   "execution_count": null,
   "outputs": []
  }
 ]
}
